import sklearn
from sklearn import datasets
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction import DictVectorizer
from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer
from sklearn.preprocessing import MinMaxScaler, StandardScaler
from sklearn.feature_selection import VarianceThreshold
from scipy.stats import pearsonr
import jieba
import pandas as pd


iris = datasets.load_iris()

#导入数据集，iris接收
#数据集是字典类型

def dic_pri():
    print("数据集：", iris)
    print("数据集描述：\n", iris["DESCR"])
    x_train, x_test, y_train, y_test = train_test_split(iris.data, iris.target, test_size=0.25, random_state=22)   ##特征值和目标值划分
    #训练集测试集划分,测试集占比0.25,选择一个随机数
    print("训练集特征值：\n", x_train, x_train.shape)
    return None


#对字典数据进行特征值化
#字典特征提取
def dic_demo():
    data = [{'city': 'beijing','temperature':100 },
            {'city': 'shanghai','temperature':60},
            {'city': 'shenzhen','temperature':30}]
    
    transfer = DictVectorizer(sparse = False)   
    #sparse稀疏矩阵
    #实例化转换器
    data_new = transfer.fit_transform(data)
    #调用fit_transform()
    print("data_new:\n",data_new)
    #将特征中类别信息处理成one hot编码（0、1矩阵）

    return None


#文本特征提取
def count_demo():
    data = ["life is short,i like like python", 
            "life is too long, i dislike python"]
#1. 实例化一个转换器类
    transfer = CountVectorizer()
#2. 调用fit_transform()
    data_new = transfer.fit_transform(data)
    print(data_new)
    return None


#使用jieba库对句子进行单词划分
def cut_word(text):
    return " ".join(list(jieba.cut(text)))


#1. 将中文文本特征抽取并进行分词
def count_chinese_demo2():
    data = ["一种还是几种今天很残酷，明天更残酷，后天很美好"]
    data_new = []
    for sent in data:
        data_new.append(cut_word(sent))
    #print(data_new)
#2. 实例化一个转换器类
    transfer = CountVectorizer()
#3. 调用fit_transform()
    data_final = transfer.fit_transform(data_new)
    print(data_final.toarray())
    print(transfer.get_feature_names_out())
#toarray转换为矩阵形式
#get_name获得特征名字
    return None

#用tfidf进行文本特征抽取
#tfidf数值越大的越具有分类意义 
def tfidf_demo():
    data = ["一种还是几种今天很残酷，明天更残酷，后天很美好"]
    data_new = []
    for sent in data:
        data_new.append(cut_word(sent))
    #print(data_new)
#2. 实例化一个转换器类
    transfer = TfidfVectorizer()
#3. 调用fit_transform()
    data_final = transfer.fit_transform(data_new)
    print(data_final.toarray())
    print(transfer.get_feature_names_out())
#toarray转换为矩阵形式
#get_name获得特征名字
    return None




"""
对数据进行归一化
"""
def minmax_demo():
#1. 获取数据
    data = pd.read_csv("dating.txt")
    data = data.iloc[:,:3]
    #print(data)

#2. 实例化一个转换器
    transfer = MinMaxScaler()

#3. 调用fit transform
    data_new = transfer.fit_transform(data)
    print(data_new)
    return None


"""
对数据进行归一化
"""
def stand_demo():
    data = pd.read_csv("dating.txt")
    data = data.iloc[:,:3]
    #print(data)
    transfer = StandardScaler()
    data_new = transfer.fit_transform(data)
    print(data_new)
    return None

def variance_demo():
    """
    过滤低方差特征
    """
    #1. 获取数据
    data = pd.read_csv("factor_returns.csv")
    data = data.iloc[:,1:-2]
    #print(data)
    #2. 实例化一个转换器
    transfer = VarianceThreshold(threshold=5)
    #3. 调用fit_transform
    data_new = transfer.fit_transform(data)
    print(data_new, data_new.shape)

    #计算某两个变量间相关系数
    r = pearsonr(data["pe_ratio"], data["pb_ratio"])
    print("相关系数：",r)
    return None
    


if __name__ == "__main__":
    #tfidf_demo()
    #minmax_demo()
    #stand_demo()
    variance_demo()